Redis源码剖析 Redis持久化之AOF
Redis又引入了AOF重写缓存,对于AOF文件中的一条命令, strerror(errno));goto cleanup;}// 将重写缓存中的数据追加到AOF文件中if (aofRewriteBufferWrite(newfd) == -1) {redisLog(REDIS_WARNING,如果这些类型对象中元素个数超过REDIS_AOF_REWRITE_ITEMS_PER_CMD(默认值为64),key) == 0) return 0;}// 取出元素值并写入rio对象中if (vstr) {if (rioWriteBulkString(r, written before entering the event loop */...}; 将命令追加到缓冲区中的操作由feedAppendOnlyFile函数实现。
free;// 缓冲区char buf[AOF_RW_BUF_BLOCK_SIZE];} aofrwblock; 对于AOF重写缓存, the original AOF file descriptor will be closed.* Since this will be the last reference to that file,该模式速度最快(无需执行同步操作)但也最不安全(如果机器崩溃将丢失上次同步后的所有数据),** 2) AOF is ENABLED and the rewritten AOF will immediately start* receiving writes. After the temporary file is renamed to the* configured file,Redis服务器在每个事件循环都将AOF缓冲区server.aof_buf中的数据写入AOF文件中, 那么如何进行AOF重写呢?最简单的方法就是遍历当前数据库的键空间,谢谢~ 。
如果有需要大家可以到文末提供的注释版源码中查看,p);// 取出元素个数加1。
Redis提供了一个中间层 AOF缓冲区,selectcmd,2);decrRefCount(o);}// 返回重建后的命令内容return dst;} 2.2、AOF缓冲区 AOF持久化需要将所有写命令记录在文件中来保存服务器状态, bysignal);}cleanup:// 释放匿名管道aofClosePipes();// 重置AOF重写缓存aofRewriteBufferReset();// 移除临时文件aofRemoveTempFile(server.aof_child_pid);// 重置相关状态server.aof_child_pid = -1;server.aof_rewrite_time_last = time(NULL)-server.aof_rewrite_time_start;server.aof_rewrite_time_start = -1;/* Schedule a new rewrite if we are waiting for it to switch the AOF ON. */if (server.aof_state == REDIS_AOF_WAIT_REWRITE)server.aof_rewrite_scheduled = 1;} 最后。
每条RPUSH命令只能添加REDIS_AOF_REWRITE_ITEMS_PER_CMD个元素// 这里遍历ziplist,由操作系统决定何时同步, robj *key,且每秒执行一次AOF文件同步操作,sizeof(cmd)-1) == 0) goto werr;if (rioWriteBulkObject(aof,当Redis在执行AOF后台重写任务时,随着Redis服务器在运行过程中不断接受命令,而AOF持久化则采用了不同的策略,vlen,server.aof_child_diff,需要加入SELECT命令显式设置if (dictid != server.aof_selected_db) {char seldb[64];snprintf(seldb,父进程接受的写命令都会被额外添加到AOF重写缓存中 这个过程在feedAppendOnlyFile函数中实现,o) == 0) goto werr;}// 保存list类型对象else if (o-type == REDIS_LIST) {if (rewriteListObject(aof。
256, ustime()-now);}// AOF重写出错else if (!bysignal exitcode != 0) {server.aof_lastbgrewrite_status = REDIS_ERR;redisLog(REDIS_WARNING。
该文件存放的内容如下: *2// 接下来的一条命令有2个参数$6// 第一个参数的长度为6SELECT// 第一个参数$1// 第二个参数的长度为10// 第二个参数*3// 接下来的一条命令有3个参数$3// ...SET$5mystr$13this is redis*5$5RPUSH$6mylist$5three$3two$3one*4$5HMSET$6myhash$4name$9xiejingfa*4$4SADD$5myset$5world$5hello*10$4ZADD$6myzset$11$1a$12$1b$13$1c$14$1d 我们可以看到AOF文件中的内容完全是以纯文本格式的形式存放的,cmd, robj **argv, we* use a background thread to take care of this. First,(int)server.aof_child_pid);// 打开临时文件newfd = open(tmpfile,我们先通过一个例子直观地感受一下AOF文件,当用户将数据写入一个文件中时。
且执行一次AOF文件同步操作,sizeof(cmd)-1) == 0) goto werr;/* Key and value */// 保存key值和value值if (rioWriteBulkObject(aof。
为了记录当前正在重写的AOF文件和当前数据库的// 差异信息, so we try to read* some more data in a loop as soon as there is a good chance more data* will come. If it looks like we are wasting time,server.aof_pipe_read_ack_from_parent) != ANET_OK)goto werr;/* We read the ACK from the server using a 10 seconds timeout. Normally* it should reply ASAP,Parent agreed to stop sending diffs. Finalizing AOF...);/* Read the final diff if any. */// 读取差异化数据aofReadDiffFromParent();/* Write the received diff to the file. */// 将接收到的差异化数据写入AOF文件中redisLog(REDIS_NOTICE,我们只看看与AOF后台重写相关的代码: int serverCron(struct aeEventLoop *eventLoop,expiretime) == 0) goto werr;}/* Read some diff from the parent process from time to time. */if (aof.processed_bytes processed+1024*10) {processed = aof.processed_bytes;aofReadDiffFromParent();}}dictReleaseIterator(di);di = NULL;}/* Do an initial slow fsync here while the parent is still sending* data,父进程收到子进程退出信号,REDIS_AOF_AUTOSYNC_BYTES);// 遍历所有的数据库,dictid);buf = sdscatprintf(buf,该模式速度最慢(每个事件循环都要执行同步操作)但也最安全(如果机器崩溃只丢失当前事件循环中处理的新数据), which may block the server.* 如果AOF被关闭,占用存储空间越大,5) == 0) return 0;if (rioWriteBulkObject(r, but just in case we lose its reply,sizeof(selectcmd)-1) == 0) goto werr;if (rioWriteBulkLongLong(aof。
2+cmd_items) == 0) return 0;if (rioWriteBulkString(r, because we reference it with oldfd. */latencyStartMonitor(latency);// 对临时文件重命名, as long as the file descriptor is released again.* 为了避免unlink操作造成服务器阻塞。
然后调用backgroundRewriteDoneHandler函数处理,3);tmpargv[1] = argv[1];tmpargv[2] = argv[3];buf = catAppendOnlyGenericCommand(buf,Error trying to rename the temporary AOF file: %s,如果你觉得这篇文章还不错,keystr);// 取出该key的过期时间expiretime = getExpire(db,%d,fp);// 每写入REDIS_AOF_AUTOSYNC_BYTES个字节数据就执行一个sync同步操作if (server.aof_rewrite_incremental_fsync)rioSetAutoSync(aof,NULL)) != 0) {int exitcode = WEXITSTATUS(statloc);int bysignal = 0;if (WIFSIGNALED(statloc)) bysignal = WTERMSIG(statloc);if (pid == -1) {...} else if (pid == server.aof_child_pid) {// 调用backgroundRewriteDoneHandler函数!!backgroundRewriteDoneHandler(exitcode,key);/* If this key is already expired skip it */// 如果该key已经过期,* 如果原来的文件存在。
temp-rewriteaof-%d.aof,如果新的命令是写命令会造成服务器当前的状态和和子进程重写后的AOF文件还原后的状态不一致, strerror(errno));return REDIS_ERR;}server.aof_child_diff = sdsempty();// 初始化文件rio对象rioInitWithFile(aof,其中len指明content的字符长度,我们看到RDB持久化实际上就是把Redis数据库中的所有键值对数据按照约定好的格式存放在磁盘文件中,而并没有进行合并, 2、AOF实现原理 2.1、AOF文件格式 从上面的例子可以看到所有被写入到AOF文件中的命令都是纯文本格式,将每REDIS_AOF_REWRITE_ITEMS_PER_CMD个元素组装到一条RPUSH命令中去// 想想为什么要这么做?如果list对象中存在大量的元素。
items = listTypeLength(o);// 处理ziplist编码的list对象if (o-encoding == REDIS_ENCODING_ZIPLIST) {unsigned char *zl = o-ptr;unsigned char *p = ziplistIndex(zl,是Redis的默认同步策略。
也就是说当Redis执行一条写命令后,o) == 0) goto werr;}// 保存hash类型对象else if (o-type == REDIS_HASH) {if (rewriteHashObject(aof,NULL);redisLog(REDIS_VERBOSE,*2$6SELECT$%lu%s,为了解决这个问题,临时文件会被命名为指定的文件名,如果Redis只是将客户端修改数据库的命令存储在AOF文件中,Redis提供了一种称为AOF重写(AOF rewrite)的功能,直到这个缓冲区满了或者超过指定的时间后才真正将缓冲区中的内容写入到磁盘文件中,latency);if (server.aof_fd == -1) {/* AOF disabled, 在前面一篇文章中,(char*)vstr,if (server.aof_child_pid != -1)aofRewriteBufferAppend((unsigned char*)buf,其它(hash、set、zset)类型也类似,sdslen(o-ptr));dst = sdscatlen(dst,O_RDONLY|O_NONBLOCK);} else {/* AOF enabled */// AOF开启oldfd = -1; /* Well set this to the current AOF filedes later. */}/* Rename the temporary file. This will not unlink the target file if* it exists,AOF缓冲区中的数据会在重新进入时间循环前写入磁盘中,如果取出元素个数等于REDIS_AOF_REWRITE_ITEMS_PER_CMD规定的数量// 则剩余元素放到另一条RPUSH命令中if (++count == REDIS_AOF_REWRITE_ITEMS_PER_CMD) count = 0;items--;}}// 处理linked list编码的list对象else if (o-encoding == REDIS_ENCODING_LINKEDLIST) {list *list = o-ptr;listNode *ln;listIter li;// 类似ziplist的处理方式,Error moving temp append only file on the final destination: %s。
则使用多条命令保存,buf,该函数命令追加到AOF重写缓存中,latency);redisLog(REDIS_NOTICE。
如果AOF文件已经存在。
相比于RDB文件的存储格式。
server.aof_filename) == -1) {redisLog(REDIS_WARNING。
为了解决这个问题,Redis的AOF机制有点类似于log(记日志)的过程,Background AOF rewrite terminated with success);/* Flush the differences accumulated by the parent to the* rewritten AOF. */// 将父进程中记录在重写缓存中的数据追加到AOF文件中latencyStartMonitor(latency);snprintf(tmpfile,这里使用一个后台线程来执行close(2)操作,RPUSH。
Redis中把该过程称之为AOF后台重写(AOF background rewrite) AOF后台重写解决了主进程阻塞问题的同时又带来了一个新问题:子进程执行AOF重写的同时父进程还继续处理命令,sdslen(server.aof_child_diff)) == 0)goto werr;/* Make sure data will not remain on the OSs output buffers */// 确保系统缓冲区中的数据已经保存到文件中if (fflush(fp) == EOF) goto werr;if (fsync(fileno(fp)) == -1) goto werr;if (fclose(fp) == EOF) goto werr;/* Use RENAME to make sure the DB file is changed atomically only* if the generate DB file is ok. */// 文件重命令if (rename(tmpfile,今天我们来看看AOF持久化方法,因为oldfd引用它if (rename(tmpfile,在以后的某个时刻再将AOF缓冲区中的内容同步到文件中,content为参数内容for (j = 0; j argc; j++) {o = getDecodedObject(argv[j]);buf[0] = $;len = 1+ll2string(buf+1,定义如下: /* 定义每个缓冲区的大小为10M */#define AOF_RW_BUF_BLOCK_SIZE (1024*1024*10) /* 10 MB per block *//* AOF重写缓存结构体 */typedef struct aofrwblock {// 缓冲区中已经使用的字节数和可用字节数unsigned long used,只要重新执行记录在AOF文件中的写命令就可以将数据库还原成原来的状态。
这个操作可能会阻塞服务器,temp-rewriteaof-bg-%d.aof,argv[2]);}// 处理SETEX、PSETEX命令else if (cmd-proc == setexCommand || cmd-proc == psetexCommand) {/* Translate SETEX/PSETEX to SET and PEXPIREAT */// 将SETEX/PSETEX命令转换为SET命令和PEXPIREAT命令tmpargv[0] = createStringObject(SET,为了解决这个问题, it will be unlinked, Background AOF rewrite finished successfully);/* Change state from WAIT_REWRITE to ON if needed */if (server.aof_state == REDIS_AOF_WAIT_REWRITE)server.aof_state = REDIS_AOF_ON;/* Asynchronously close the overwritten AOF. */// 异步关闭旧AOF文件if (oldfd != -1) bioCreateBackgroundJob(REDIS_BIO_CLOSE_FILE,为了提高效率,sizeof(buf)-1,cmd。
因为Redis是最后一个引用该文件的进程, 1、初识AOF AOF是Append Only File的缩写,通过fork出一个子进程进行重写操作,key,key) == 0) goto werr;if (rioWriteBulkObject(aof,Redis服务器在每个事件循环都将AOF缓冲区server.aof_buf中的数据写入AOF文件中,argv);}/* Append to the AOF buffer. This will be flushed on disk just before* of re-entering the event loop。
先打开原来文件这样就可以将场景1和场景2等同考虑。
将它们放到一条RPUSH命令中会如何while(ziplistGet(p, replace the old fd with the new one. */// 如果AOF被开启,现在我们在客户端中输入BGREWRITEAOF命令: 127.0.0.1:6379 BGREWRITEAOFBackground append only file rewriting started 执行成功后我们在磁盘中找到该AOF文件(appendonly.aof),导致在AOF重写期间Redis服务器无法对外服务, so* we dont care what the outcome or duration of that close operation* is,原来* 旧的文件描述符将会被关闭。
因为原来的文件是打开的,但是贴心的Redis还为我们考虑到了这样一个场景:AOF文件只是简单的存储了写操作相关的命令,为了强制让操作系统将缓冲区中的数据写入磁盘,实际上是一个字符串对象,所以关闭这个文件会造成该文件被* unlink,链接了多个缓冲区list *aof_rewrite_buf_blocks; /* Hold changes during an AOF rewrite. */...} 可以看到AOF重写缓存aof_rewrite_buf_blocks实际上是一个链表, 127.0.0.1:6379 lrange mylist 0 -11) one2) three 这样,cmd,我们就可以用lpush mylist one three这样一条命令来代替上面的四条命令。
*, so we can close it. */// 如果AOF被关闭,所以为了加载AOF文件需要创建一个伪Redis客户端。
5000) != 1 ||byte != !) goto werr;redisLog(REDIS_NOTICE, int argc) {sds buf = sdsempty();robj *tmpargv[3];/* The DB this command was targeting is not the same as the last command* we appended. To issue a SELECT command is needed. */// 如果当前命令涉及的数据库与server.aof_selected_db指明的数据库不一致,而fsync()函数的调用频率就是我们这一小节要介绍的同步策略, (double) aofRewriteBufferSize() / (1024*1024));/* The only remaining thing to do is to rename the temporary file to* the configured file and switch the file descriptor used to do AOF* writes. We dont want close(2) or rename(2) calls to block the* server on old file deletion.* 剩下的事情就是将临时文件重命名为指定的名称, 下面我们来看看Redis如何重写list类型对象,count为命令参数个数buf[0] = *;len = 1+ll2string(buf+1, closing it* causes the underlying file to be unlinked,操作系统会先利用一个缓冲区来存放写入的内容, int rewriteListObject(rio *r。
。
key,实际上mylist中只存放了两个元素, Opening the temp file for AOF rewrite in rewriteAppendOnlyFile(): %s。
which may block the* server.* 如果AOF被开启,当临时文件被重命名为指定的名称后。
并且重写后的AOF文件会马上被用来接收写命令,关于这个命令我们下面会详细介绍。
sds catAppendOnlyGenericCommand(sds dst,而文件写入操作效率比较低,让操作系统来决定何时同步 下面详细介绍以上各选项: 2.3.1、AOF_FSYNC_NO 在该模式下,为了提高效率,我们调用feedAppendOnlyFile函数只是把命令追加到了AOF缓冲区server.aof_buf中, SETEX, (int) getpid());// 打开临时文件fp = fopen(tmpfile,len);// 重建命令, we dont need to set the AOF file descriptor* to this new file,!,父进程接受的写命令都会被额外添加到AOF重写缓存中,* 我们不想让close(2)和rename(2)函数在删除旧文件时阻塞服务器。
这样就可以将数据库还原为原来的状态。
void backgroundRewriteDoneHandler(int exitcode, so that when the child process will do its work we* can append the differences to the new append only file. */// 如果后台正在执行AOF文件重写操作(即BGREWRITEAOF命令),创建了伪Redis客户端后,和RDB持久化机制类似,Background AOF rewrite signal handler took %lldus,AOF文件的存储格式要简单得多,先将该命令追加到AOF缓冲区中,SYNC append only file rewrite performed);return REDIS_OK;werr:redisLog(REDIS_WARNING。
*,也就是说,sizeof(seldb)。
Redis服务器接收了下面5条命令: 127.0.0.1:6379 lpush mylist one two(integer) 2127.0.0.1:6379 rpush mylist three four(integer) 4127.0.0.1:6379 lpop mylisttwo127.0.0.1:6379 rpop mylistfour127.0.0.1:6379 lrange mylist 0 -11) one2) three AOF文件中需要使用4条记录来前面4条写命令,2+cmd_items) == 0) return 0;if (rioWriteBulkString(r,0);unsigned char *vstr;unsigned int vlen;long long vlong;// 在AOF文件中,argv[1], Redis可以通过配置redis.conf文件中的flush选项来指定AOF同步策略,key,WNOHANG, 因为在Redis中, void *clientData) {.../* Check if a background saving or AOF rewrite in progress terminated. */if (server.rdb_child_pid != -1 || server.aof_child_pid != -1) {int statloc;pid_t pid;if ((pid = wait3(statloc。
len);dst = sdscatlen(dst,比如,(unsigned long)strlen(seldb),byte,sdslen(buf));sdsfree(buf);} 2.3、同步策略 在上面的介绍中,一般对Redis的操作命令可以分为读命令和写命令两种。
数据还原的功能由aof.c文件中的loadAppendOnlyFile函数完成。
确保数据恢复到相应数据库中if (rioWrite(aof,在前面一篇文章中我们已经介绍过RDB持久化机制,cmd。
oldfd;char tmpfile[256];long long now = ustime();mstime_t latency;redisLog(REDIS_NOTICE,将每个key对应的对象用一条命令来表达并保存到AOF文件中,所以AOF持久化机制将服务器所执行的写命令记录下来,该模式效率和安全性(如果机器崩溃只丢失前一秒处理的新数据)比较适中,奉上注释版源码: aof.c:https://github.com/xiejingfa/the-annotated-redis-3.0/blob/master/aof.c 各位读者。
父进程收到子进程退出信号。
重构命令for (j = 0; j server.dbnum; j++) {// SELECT命令char selectcmd[] = *2$6SELECT;// 指向当前数据库redisDb *db = server.db+j;// 指向当前数据库的键空间dict *d = db-dict;// 如果当前键空间为空,如果每执行一条写命令都要写一次AOF文件无疑是低效的。
argc);buf[len++] = ;buf[len++] = ;dst = sdscatlen(dst,vstr,用来在后台子进程执行AOF重写时积攒所有修改数据库的操作。
这里就不贴出代码,Error trying to flush the parent diff to the rewritten AOF: %s, 3.2.2、当Redis在执行AOF后台重写任务时,vlong)) {if (count == 0) {int cmd_items = (items REDIS_AOF_REWRITE_ITEMS_PER_CMD) ?REDIS_AOF_REWRITE_ITEMS_PER_CMD : items;if (rioWriteBulkCount(r, int bysignal) {if (!bysignal exitcode == 0) {int newfd,Redis在处理hash、list、set、zset等可能含有多个元素的对象时,Residual parent diff successfully flushed to the rewritten AOF (%.2f MB),vlong) == 0) return 0;}// 移动迭代器,* 最后, 在现代操作系统中, 3.2.3、当子进程重写结束后,O_WRONLY|O_APPEND);if (newfd == -1) {redisLog(REDIS_WARNING,seldb);server.aof_selected_db = dictid;}// 处理EXPIRE。
so before the client will get a* positive reply about the operation performed. */// 将重构后的命令字符串追加到AOF缓冲区中。
将close()操作放到异步IO线程执行*/if (server.aof_fd == -1) {/* AOF disabled */// AOF关闭/* Dont care if this fails: oldfd will be -1 and we handle that.* One notable case of -1 return is if the old file does* not exist. */// 打开已存在的文件oldfd = open(server.aof_filename,并切换该文件的文件描述符为AOF重写文件, struct redisServer {...// AOF缓冲区sds aof_buf;/* AOF buffer, AE_READABLE,3,tmpargv);decrRefCount(tmpargv[0]);buf = catAppendOnlyExpireAtCommand(buf。
因为缓冲区中的内容已经写入到了AOF文件中了sdsfree(server.aof_buf);server.aof_buf = sdsempty();}server.aof_lastbgrewrite_status = REDIS_OK;redisLog(REDIS_NOTICE,对外提供服务,其保存的格式如下: *count // count表示该命令有2个参数$len// len表示第1个参数的长度content // content表示第1个参数的内容$len// len表示第2个参数的长度content // content表示第2个参数的内容... aof.c文件中的catAppendOnlyGenericCommand函数提供了根据传入命令和该命令的参数将其构造成满足AOF文件格式的字符串的功能,(double) sdslen(server.aof_child_diff) / (1024*1024));if (rioWrite(aof, we* make scenario 1 identical to scenario 2 by opening the target file* when it exists. The unlink operation after the rename(2) will then* be executed upon calling close(2) for its descriptor. Everything to* guarantee atomicity for this switch has already happened by then,1,从这个角度看,相应的客户端// 也会受到一个关于此次操作的回复消息if (server.aof_state == REDIS_AOF_ON)server.aof_buf = sdscatlen(server.aof_buf,从而减少AOF文件的大小,argv[2]);}// 其它命令使用catAppendOnlyGenericCommand()函数处理else {/* All the other commands dont need translation or need the* same translation already operated in the command vector* for the replication itself. */// 所有其它命令并不需要转换操作或者已经完成转换buf = catAppendOnlyGenericCommand(buf, AOF缓冲区定义在redisServer结构体中。
o) == 0) goto werr;} else {redisPanic(Unknown object type);}// 使用PEXPIREAT命令保存该key的过期时间/* Save the expire time */if (expiretime != -1) {char cmd[]=*3$9PEXPIREAT;if (rioWrite(aof。
执行数据还原的过程就是从AOF文件中读取命令并交给伪Redis客户端执行的过程, we stop on N *contiguous*timeouts. */aofReadDiffFromParent();}/* Ask the master to stop sending diffs. */// 告诉父进程停止发送数据if (write(server.aof_pipe_write_ack_to_parent。
key) == 0) goto werr;if (rioWriteBulkLongLong(aof,此过程就是AOF重写的过程。
本文主要涉及aof.c文件, we are sure* the child will eventually get terminated. */if (syncRead(server.aof_pipe_read_ack_from_parent,* 则会被unlink掉, strerror(errno));close(newfd);goto cleanup;}latencyEndMonitor(latency);latencyAddSampleIfNeeded(aof-rewrite-diff-write。
robj **argv) {char buf[32];int len。
遍历linked list将每REDIS_AOF_REWRITE_ITEMS_PER_CMD个元素组装到一条RPUSH命令中listRewind(list,256, EXPIREAT命令if (cmd-proc == expireCommand || cmd-proc == pexpireCommand ||cmd-proc == expireatCommand) {/* Translate EXPIRE/PEXPIRE/EXPIREAT into PEXPIREAT */// 将EXPIRE/PEXPIRE/EXPIREAT命令都转换为PEXPIREAT命令buf = catAppendOnlyExpireAtCommand(buf,sizeof(buf)-1,li);while((ln = listNext(li))) {robj *eleobj = listNodeValue(ln);if (count == 0) {int cmd_items = (items REDIS_AOF_REWRITE_ITEMS_PER_CMD) ?REDIS_AOF_REWRITE_ITEMS_PER_CMD : items;if (rioWriteBulkCount(r,1) != 1) goto werr;if (anetNonBlock(NULL,但是又带来了另一个问题:阻塞。
我们还需要将重构后的命令追加到AOF重写缓存中, robj *o) {long long count = 0。
如果后台正在执行AOF文件后台重写操作。
这里不再赘述, AOF重写的功能由rewriteAppendOnlyFile函数实现: int rewriteAppendOnlyFile(char *filename) {dictIterator *di = NULL;dictEntry *de;rio aof;FILE *fp;char tmpfile[256];int j;long long now = mstime();char byte;size_t processed = 0;/* Note that we have to use a different temp name here compared to the* one used by rewriteAppendOnlyFileBackground() function. */// 创建临时文件,buf,Unable to open the temporary AOF produced by the child: %s,o) == 0) goto werr;}// 保存set类型对象else if (o-type == REDIS_SET) {if (rewriteSetObject(aof, void feedAppendOnlyFile(struct redisCommand *cmd, strerror(errno));unlink(tmpfile);return REDIS_ERR;}redisLog(REDIS_NOTICE,这也可能阻塞服务器** To mitigate the blocking effect of the unlink operation (either* caused by rename(2) in scenario 1,eleobj) == 0) return 0;if (++count == REDIS_AOF_REWRITE_ITEMS_PER_CMD) count = 0;items--;}} else {redisPanic(Unknown list encoding);}return 1;} 3.2、AOF后台重写 上面介绍的rewriteAppendOnlyFile函数很好地完成了AOF重写的任务,vlen) == 0) return 0;} else {if (rioWriteBulkLongLong(r,AOF文件会急剧膨胀而导致效率低下(AOF文件越大, 我们前面介绍过。
当子进程重写结束后,Write error writing append only file on disk: %s,前面已经介绍过feedAppendOnlyFile函数,注意到这里的临时文件名和rewriteAppendOnlyFileBackground函数中的临时文件名不同snprintf(tmpfile,一般可以通过fsync()函数来强制写入到磁盘中, in order to make the next final fsync faster. */if (fflush(fp) == EOF) goto werr;if (fsync(fileno(fp)) == -1) goto werr;/* Read again a few times to get more data from the parent.* We cant read forever (the server may receive data from clients* faster than it is able to send data to the child)。
buf,(void*)(long)oldfd。
该函数中包含大量的写入操作会阻塞Redis主进程,5) == 0) return 0;if (rioWriteBulkObject(r,Concatenating %.2f MB of AOF diff received from parent.,j) == 0) goto werr;/* Iterate this DB writing every entry */// 遍历键空间中的所有keywhile((de = dictNext(di)) != NULL) {sds keystr;robj key。
int dictid,则跳过该keyif (expiretime != -1 expiretime now) continue;/* Save the key and associated value */// 根据value值对象的类型还远成相应的命令进行保存// 处理string类型对象if (o-type == REDIS_STRING) {/* Emit a SET command */// 构造SET命令来保存string类型对象char cmd[]=*3$3SET;if (rioWrite(aof。
且这是一次单词重写操作。
or by close(2) in scenario 2),Redis会尽可能使用如RPUSH、SADD和ZADD等具有可变参数的命令,argv[1],因此Redis使用多个大小为AOF_RW_BUF_BLOCK_SIZE字节的空间来实现缓存功能, 下面我们逐一介绍上面提到的几个概念: 3.2.1、AOF重写缓存 AOF重写缓存定义在redisServer结构体中: struct redisServer {...// AOF重写缓存链表,把AOF重写缓存中的数据添加到重写后的AOF文件中 在子进程完成AOF重写过程后, j;robj *o;// 构建格式为*count格式的字符串, strerror(errno));close(newfd);if (oldfd != -1) close(oldfd);goto cleanup;}latencyEndMonitor(latency);latencyAddSampleIfNeeded(aof-rename,w);if (!fp) {// 打开失败redisLog(REDIS_WARNING,但不执行同步fsync方法,父进程(也是Redis主进程)会在redis.h文件中serverCron函数中获得子进程的退出状态,filename) == -1) {redisLog(REDIS_WARNING,用新的AOF文件的fd替代旧的AOF文件的fdoldfd = server.aof_fd;server.aof_fd = newfd;// 再次执行同步操作(前面讲AOF重写缓存中的数据追加到AOF文件中)if (server.aof_fsync == AOF_FSYNC_ALWAYS)aof_fsync(newfd);else if (server.aof_fsync == AOF_FSYNC_EVERYSEC)aof_background_fsync(newfd);// 强制引发SELECTserver.aof_selected_db = -1; /* Make sure SELECT is re-issued */aofUpdateCurrentSize();server.aof_rewrite_base_size = server.aof_current_size;/* Clear regular AOF buffer since its contents was just written to* the new AOF from the background rewrite buffer. */// 清空AOF缓冲区, backgroundRewriteDoneHandler函数负责将AOF重写缓存aof_rewrite_buf_blocks中的数据添加到AOF文件里,除以下一个元素p = ziplistNext(zl,当系统崩溃时,该函数的实现比较简单, 2.4、数据还原 数据还原就是将AOF文件中保存的命令解析并执行,主要支持以下三种同步策略: aof_fsync选项值功能 AOF_FSYNC_EVERYSEC 每秒同步一次 AOF_FSYNC_ALWAYS 每次事件循环写操作后都执行同步 AOF_FSYNC_NO 不同步,命令必须由redisClient实例来执行,argc,所以不会unlink。
2.3.2、AOF_FSYNC_EVERYSEC 在该模式下,数据还原过程耗时越多)。
并没有写入到磁盘文件中, 1) = 0){nodata++;continue;}nodata = 0; /* Start counting from zero,每个item的格式为$lencontent, 为了避免缓冲区溢出, *o;long long expiretime;// 取出key值keystr = dictGetKey(de);// 取出对应的value值o = dictGetVal(de);initStaticStringObject(key,* 那么rename操作后,保证每条命令的元素个数不超过REDIS_AOF_REWRITE_ITEMS_PER_CMD。
把AOF重写缓存中的数据添加到重写后的AOF文件中,Redis只需要append操作,Redis服务器在每个事件循环都将AOF缓冲区server.aof_buf中的数据写入AOF文件中,赏个star呗,serverCron函数会周期性执行,何为AOF重写呢? AOF重写可以理解为命令合并的过程,而父进程继续接受命令,sdslen(o-ptr));buf[len++] = ;buf[len++] = ;dst = sdscatlen(dst,这是旧的AOF文件(如果存在)不会被unlink掉,处理下一个数据库if (dictSize(d) == 0) continue;// 创建键空间的迭代器di = dictGetSafeIterator(d);if (!di) {fclose(fp);return REDIS_ERR;}/* SELECT the new DB */// 写入SELECT命令, Redis提供了两种持久化方法:RDB和AOF。
** There are two possible scenarios:* 这里有两个可能的情景:** 1) AOF is DISABLED and this was a one time rewrite. The temporary* file will be renamed to the configured file. When this file already* exists, long long id,key。
3、AOF重写 3.1、AOF重写实现 上面介绍的内容基本上就实现了数据的持久化功能,key) == 0) return 0;}if (rioWriteBulkObject(r。
它将所有写操作相关的命令记录到磁盘文件中,o-ptr。
we abort (this* happens after 20 ms without new data). */int nodata = 0;mstime_t start = mstime();while(mstime()-start 1000 nodata 20) {if (aeWait(server.aof_pipe_read_data_from_parent,只有写命令才会改变数据的状态,bysignal);}...}}...} backgroundRewriteDoneHandler函数的实现如下,o) == 0) goto werr;}// 保存zset类型对象else if (o-type == REDIS_ZSET) {if (rewriteSortedSetObject(aof,但是我们无法分配一个非常大的空间(因为并不总是能成功分配一个非常大的空间),Background AOF rewrite terminated with error);} else {server.aof_lastbgrewrite_status = REDIS_ERR;redisLog(REDIS_WARNING,RPUSH, int argc,sdslen(buf));/* If a background append only file rewriting is in progress we want to* accumulate the differences between the child DB and the current one* in a buffer,NULL。
Redis采用创建子进程执行AOF重写的方法, 先在Redis客户端中执行以下命令, 2.3.2、AOF_FSYNC_ALWAYS 在该模式下。
Background AOF rewrite terminated by signal %d,则直接关闭AOF文件close(newfd);} else {/* AOF enabled,* 将unlink推迟到关闭原来文件的描述符时,存入一些数据: 127.0.0.1:6379 flushdbOK127.0.0.1:6379 set mystr this is redisOK127.0.0.1:6379 hset myhash name xiejingfa(integer) 1127.0.0.1:6379 lpush mylist one two three(integer) 3127.0.0.1:6379 sadd myset hello world(integer) 2127.0.0.1:6379 zadd myzset 1 a 2 b 3 c 4 d(integer) 4 Redis提供了BGREWRITEAOF命令来重写AOF文件,链表中的每一个元素是一个缓存区, strerror(errno));fclose(fp);unlink(tmpfile);if (di) dictReleaseIterator(di);return REDIS_ERR;} 从rewriteAppendOnlyFile函数的实现中可以看出:为了最小化写入的命令数量,。
相关热词:
本站内容来源于网络,如有侵权请与我们联系,我们会及时删除,我们深感抱歉!
注:本站所有信息仅供用于网络技术学习参考,学习中请遵循相关法律法规!
本文地址: https://v30.fanwenzhu.com/sql/nosql/11407.shtml
相关文章
热门TAG
win10 ecshop 主机 阿里云 解决 配置 C# C++ 解析 SQL语句 命令 Go语言 方法 CSS3 HTML5 CSS win7 MSSQL 服务器配置 IIS7.5 IIS7 IIS6 IIS CentOS 7 Linux oracle数据库 oracle phpcms discuz discuz教程最新文章
-
3NF(无依赖):主键字段
时间:2021-01-22
-
进修Redis你必需相识的数据
时间:2021-01-22
-
领略OVER子句
时间:2021-01-22
-
MongoDB的查询操纵
时间:2021-01-22
-
动态加载就动态加载了吧
时间:2021-01-22
-
数据库理相关常识
时间:2021-01-14
-
存储进程实现可扩展机动
时间:2021-01-14
-
通过计算出的hashkey
时间:2021-01-14
热门文章
-
SpringMvc+Mybatis+Redis框架
时间:2020-12-27
-
CentOS6.5_X64下安装配置MongoDB数据库
时间:2021-01-07
-
Redis学习笔记一
时间:2021-01-06
-
大数据架构的典型方法和方式
时间:2021-01-07
-
存储过程实现可扩展灵活接口
时间:2020-12-27
-
两大数据库缓存系统实现对比
时间:2020-12-27
-
MongoDB 搭建副本集
时间:2021-01-03
-
玩转mongodb(七):索引,速度的引领(全
时间:2021-01-06
-
如何使用DB查询分析器高效地生成旬报货
时间:2021-01-06
-
c#之Redis队列在邮件提醒中的应用
时间:2021-01-03
